# -*- coding: utf-8 -*-
import re

from src.com.bsjs.util.base_util import BaserUtil
from src.com.bsjs.util.file_util import FileClient
from src.com.bsjs.util.html_util import HtmlClient
from src.com.bsjs.util.http_util import HttpClient
from src.com.bsjs.util.redis_util import RedisUtil
from src.com.bsjs.util.selenium_util import SeleniumClient
from src.com.bsjs.util.sql_util import MySQLUtils

def extract_chapter_number(text):
    pattern = r'第(\d+)章'
    match = re.search(pattern, text)
    if match:
        return match.group(1)
    else:
        return None

class Spider:
    def __init__(self):
        self.http_client = HttpClient()
        self.redis_client = RedisUtil()
        self.base_client = BaserUtil()
        self.mysql_client = MySQLUtils()
        self.html_client = HtmlClient()
        self.file_client = FileClient()
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36"
        }
        self.selenium_client = SeleniumClient(load_selenium=False)
        self.url='https://hscode.keep1.net/'
        self.pre='https://hscode.keep1.net/'
        self.increment_key='param_item_key'

    def load_html(self):
        index_html=self.redis_client.get_value(self.url)
        if index_html!=None and len(index_html)>0:
            return index_html
        self.selenium_client.open_url(URL=self.url,fast_falg=True)
        index_html=self.selenium_client.source_code()
        self.redis_client.set_value(self.url,index_html)
        return index_html

    def load_ele(self):
        index_html=self.load_html()
        content_html=self.html_client.to_str(self.html_client.find_element(html=index_html,xpath_exp="//div[@class='hsSearchList']"))
        elements=self.html_client.find_elements(html=content_html,xpath_exp="//dl[not(ancestor::dl)]")

        ele_dicts=[]
        for ele in elements:
            data_node=DataNode()
            ele_str=self.html_client.to_str(ele)
            title_ele=self.html_client.find_element(html=ele_str, xpath_exp="//dt")

            view_text=self.html_client.extarct_view_text(title_ele)
            detail_url_ele=self.html_client.find_element(html=self.html_client.to_str(title_ele), xpath_exp="//a")
            detail_url = None if detail_url_ele==None else detail_url_ele.attrib['href']
            detail_url=self.pre+detail_url if detail_url!=None and len(detail_url)>0 else None
            data_node.raw_text=view_text.split("\n")[0]
            data_node.detail_url=detail_url

            self.parse_sub_node(data_node,ele_str)

            ele_dicts.append(data_node)
        return ele_dicts

    def parse_sub_node(self, parent_node, ele_str):
        if ele_str == None or len(ele_str) == 0:
            return
        content_ele=self.html_client.find_element(html=ele_str, xpath_exp="//dd[not(ancestor::dd)]")
        if content_ele == None:
            return
        content_html=self.html_client.to_str(content_ele)
        sub_elements=self.html_client.find_elements(html=content_html,xpath_exp="//dl[not(ancestor::dl)]")
        if sub_elements==None or len(sub_elements)==0:
            self.parse_sub_nodeLeaf(parent_node,ele_str)
            return

        sub_nodes=[]
        for sub_element in sub_elements:
            sub_data_node = DataNode()
            sub_ele_str=self.html_client.to_str(sub_element)

            title_ele = self.html_client.find_element(html=sub_ele_str, xpath_exp="//dt")
            view_text = self.html_client.extarct_view_text(title_ele)
            detail_url_ele = self.html_client.find_element(html=self.html_client.to_str(title_ele), xpath_exp="//a")
            detail_url = None if detail_url_ele == None else detail_url_ele.attrib['href']
            detail_url = self.pre + detail_url if detail_url != None and len(detail_url) > 0 else None
            sub_data_node.raw_text = view_text.split("\n")[0]
            sub_data_node.detail_url = detail_url
            self.parse_sub_node(sub_data_node,sub_ele_str)

            sub_nodes.append(sub_data_node)
        parent_node.subNodes=sub_nodes

    def parse_sub_nodeLeaf(self, parent_node, ele_str):
        sub_elements = self.html_client.find_elements(html=ele_str, xpath_exp="//dd[not(ancestor::dd)]")
        sub_nodes = []
        for sub_element in sub_elements:
            sub_ele_str=self.html_client.to_str(sub_element)
            sub_data_node = DataNode()
            title_ele = self.html_client.find_element(html=sub_ele_str, xpath_exp="//span")
            if title_ele==None:
                continue
            view_text = self.html_client.extarct_view_text(self.html_client.to_str(title_ele))
            detail_url_ele = self.html_client.find_element(html=sub_ele_str, xpath_exp="//a")
            detail_url = None if detail_url_ele == None else detail_url_ele.attrib['href']
            sub_data_node.raw_text = view_text.split("\n")[0]
            sub_data_node.detail_url = detail_url
            sub_nodes.append(sub_data_node)
        parent_node.subNodes = sub_nodes

    def transfy_sql(self, subNodes,pid,sqls=[]):
        if subNodes==None or len(subNodes)==0:
            return sqls
        table='param_item'
        columns='id,pid,name,hscode,url'
        for node in subNodes:
            id=self.redis_client.increment_value(self.increment_key)
            hscode=extract_chapter_number(node.raw_text)
            hscode=hscode if hscode!=None and len(hscode)>0 else ''
            values = "'" + str(id) + "'" + "," \
                     + "'" + str(pid) + "'" + "," \
                     + "'" + node.raw_text + "'" + "," \
                     + "'" + hscode + "'" + "," \
                     + "'" + node.detail_url + "'"
            sql = f"INSERT INTO {table} ({columns}) VALUES ({values}) ;"
            sqls.append(sql)
            secNodes=node.subNodes
            if secNodes!=None and len(secNodes)>0:
                self.transfy_sql(subNodes=secNodes,pid=id,sqls=sqls)

        return sqls



class DataNode:
    def __init__(self):
        self.raw_text=None
        self.detail_url=None
        self.subNodes=None  #子节点链表

    def __str__(self):
        children_str = ""  if self.subNodes==None else ", ".join(str(child) for child in self.subNodes)
        return f"Node(raw_text={self.raw_text}, detail_url={self.detail_url}),\n subNode={children_str}\n"


def main():
    this = Spider()
    eles=this.load_ele()
    sqls=[]
    for ele in eles:
        sqls+=this.transfy_sql(ele.subNodes,0)
    sql_script="\n".join(sqls)
    this.file_client.write('./item.sql',line=sql_script)


if __name__ == '__main__':
    main()